#coding=utf-8
import requests
import time
import csv
import pandas as pd
import logging
from apscheduler.schedulers.blocking import BlockingScheduler
import psutil
import os

class draker(object):

    def __init__(self):
        self.TimeID = '0'
        self.headers1 ={
                        'Host': 'auth.draker.us',
                        'Referer': 'https://www.draker.us/',
                        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
                        }
        self.headers2 = {
                            'Host': 'api.draker.us',
                            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
                            'Accept': '*/*',
                            'Accept-Encoding': 'gzip, deflate, sdch, br',
                            'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
                            'Connection': 'keep-alive',
                            'Content-Type': 'application/x-www-form-urlencoded',
                            'Origin': 'https://www.draker.us',
                        }
        self.loginData = {'email': 'kangdh@cnegroup.com', 'password': 'cwp@0182'}
        self.token = []
        self.itemName1 = []
        self.itemName2 = []
        self.itemName3 = []
        self.itemName4 = []
        self.itemName5 = []
        self.itemName6 = []
        self.itemName7 = []
        self.content = {}
        self.siteID=[1869,1874,1875,2212,1328,2000,2049]
        self.siteName = ['Duke Building 129','Duke Building 87','Duke Building 98',
                         'HMV Minster I','Jefferson Solar Park','Johnston Solar I',
                         'North Kingstown Solar - Phase I']
        self.wrongSite = []

    def login(self):
        r = requests.post("https://auth.draker.us/",data=self.loginData,headers = self.headers1)
        logging.info('login successful!url:https://auth.draker.us/')
        self.token.append(r.headers['x-access-token'])

    def parse(self,token,item,siteID):
        self.headers2['x-access-token'] = token
        currentDate = time.localtime(time.time())
        currentDate = time.strftime('%Y-%m-%d %H:%M:%S', currentDate)
        currentTime = float(self.TimeID) - 28800.0
        currentTime = time.localtime(currentTime)
        currentTime = time.strftime('%Y-%m-%d %H:%M:%S', currentTime)
        self.content['scrapeTime'] = [currentDate]
        self.content['UTCdataTime'] =[currentTime]
        try:
            r = requests.get('https://api.draker.us/v1/projects/'+str(siteID) + '/datasets',headers = self.headers2)
            # logging.info('request successful,url:https://api.draker.us/v1/projects/%s/datasets'%str(siteID))
        except:
            logging.info('request error,retrying...')
            print 'retrying first link'
            r = requests.get('https://api.draker.us/v1/projects/'+str(siteID) + '/datasets', headers=self.headers2)
        try:
            for jsonItem in r.json():
                url = 'https://api.draker.us/v1/projects/'+str(siteID) + '/datasets'+ '/'+str(jsonItem['id']) + '/'+self.TimeID
                try:
                    r2 = requests.get(url,headers = self.headers2)
                    # logging.info('request successful,urlID:%s'%str(jsonItem['id']))
                    try:
                        if r2.json():
                            # logging.info('json item is not None')
                            # print len(r2.json())
                            num = 0
                            for j in r2.json():
                                if j['value'] is None:
                                    j['value'] = 0
                                for i in item:
                                    if i[1] == str(j['system_variable_id']):
                                        self.content[i[0]] =[j['value']]
                        else:
                            print 'json item is None'
                            logging.info('json item is None')
                    except:
                        print 'json item is None zzz'
                        logging.info('cannot read this json item,prepare to retry.jsonItem id:%s'%str(jsonItem['id']))
                        self.wrongSite.append(url)
                except:
                    print 'request error,retrying...'
                    logging.info('request error,prepare retry.prepare to retry')
                    self.wrongSite.append(url)
        except:
            self.itemName1 = []
            self.itemName2 = []
            self.itemName3 = []
            self.itemName4 = []
            self.itemName5 = []
            self.itemName6 = []
            self.itemName7 = []
            print 'cannot handhold the website,waiting nexttime'
            logging.info('cannot handhold the website,waiting nexttime')

    def readData(self):
        reader1 = csv.reader(open('DukeBuilding129.csv','r'))
        for line in reader1:
            self.itemName1.append(line)
        reader2 = csv.reader(open('DukeBuilding87.csv', 'r'))
        for line in reader2:
            self.itemName2.append(line)
        reader3 = csv.reader(open('DukeBuilding98.csv', 'r'))
        for line in reader3:
            self.itemName3.append(line)
        reader4 = csv.reader(open('HMVMinsterI.csv', 'r'))
        for line in reader4:
            self.itemName4.append(line)
        reader5 = csv.reader(open('JeffersonSolarPark.csv', 'r'))
        for line in reader5:
            self.itemName5.append(line)
        reader6 = csv.reader(open('JohnstonSolarI.csv', 'r'))
        for line in reader6:
            self.itemName6.append(line)
        reader7 = csv.reader(open('NorthKingstownSolar-PhaseI.csv', 'r'))
        for line in reader7:
            self.itemName7.append(line)

    def writeCsv(self,item,siteName):
        currentTime = float(self.TimeID)
        currentDate = time.localtime(currentTime)
        currentDate = time.strftime('%Y-%m-%d %H:%M:%S', currentDate)
        currentDate = currentDate.replace(' ',',').replace(':','-')
        df = pd.DataFrame(item,columns=item.keys(),index=None)
        siteName = siteName.replace(' ','')
        Name = siteName+'-'+currentDate
        # print df
        df.to_csv('F:\python\%s\%s.csv'%(siteName,Name),index=False)

    def wrongSiteRetry(self,item):
        if self.wrongSite is not None:
            for url in self.wrongSite:
                logging.info('retrying url:%s'%url)
                for times in range(1,6):
                    print 'now retrying times:',times
                    try:
                        r2 = requests.get(url, headers=self.headers2)
                        logging.info('retrying times:%d'%times)
                        logging.info('retrying successful url:%s'%url)
                        try:
                            if r2.json():
                                print len(r2.json())
                                for j in r2.json():
                                    if j['value'] is None:
                                        j['value'] = 0
                                    for i in item:
                                        if i[1] == str(j['system_variable_id']):
                                            self.content[i[0]] = [j['value']]
                                print 'retrying successful'
                                logging.info('retrying successful,now continue to work')
                                break
                            else:
                                print 'retry json item is None'
                                logging.info('retry json item is None')
                        except:
                            print 'retry json item is None zzz'
                            logging.info('retry cannot request json')
                    except:
                        print 'cannot request this url:%s'%url
                        logging.info('cannot request this url:%s,need help!!!!!'%url)
        else:
            print 'do not need retry'
            logging.info('error site is None,good job!')

    def obtainTime(self,token,siteID):
        url = 'https://api.draker.us/v1/projects/'+str(siteID)
        self.headers2['x-access-token'] = token
        try:
            r = requests.get(url,headers=self.headers2)
            # logging.info('request successful, get the UTC time')
            UTCTime = r.json()['last_calculated_at']
            processedTime = [int(UTCTime[:4]),int(UTCTime[5:7]),int(UTCTime[8:10]),int(UTCTime[11:13]),
                            int(UTCTime[14:16]),int(UTCTime[17:18]),0,0,0]
            currentTime = time.mktime(processedTime)+28800
            self.TimeID = str(currentTime)[:-2]

        except:
            print 'The acquisition time failed'
            logging.info('request error, do not get the UTC time')

    def start(self):
        time1 = time.time()
        logging.basicConfig(filename='drakerSpider.log', filemode='w', level=logging.INFO)
        logging.info('Started')
        self.readData()
        # print len(self.itemName1)
        # logging.info('get %s item from DukeBuilding129.csv'%len(self.itemName1))
        # print len(self.itemName2)
        # logging.info('get %s item from DukeBuilding87.csv' % len(self.itemName2))
        # print len(self.itemName3)
        # logging.info('get %s item from DukeBuilding98.csv' % len(self.itemName3))
        # print len(self.itemName4)
        # logging.info('get %s item from HMVMinsterI.csv' % len(self.itemName4))
        # print len(self.itemName5)
        # logging.info('get %s item from JeffersonSolarPark.csv' % len(self.itemName5))
        # print len(self.itemName6)
        # logging.info('get %s item from JohnstonSolarI.csv' % len(self.itemName6))
        # print len(self.itemName7)
        # logging.info('get %s item from NorthKingstownSolar-PhaseI.csv' % len(self.itemName7))
        self.login()
        time.sleep(2)
        itemName =[self.itemName1,self.itemName2,self.itemName3,self.itemName4,self.itemName5,self.itemName6,self.itemName7]
        for i,j,a in zip(self.siteID,self.siteName,itemName):
            self.obtainTime(token=self.token[0],siteID=i)
            print 'lastcalculated time is:', self.TimeID
            self.parse(token=self.token[0],item=a,siteID=i)
            print 'wrongSite number:',len(self.wrongSite)
            logging.info('wrongSite number is :%s'%len(self.wrongSite))
            self.wrongSiteRetry(item=a)
            self.wrongSite = []
            self.content['stationID'] = j.replace(' ','')
            time.sleep(2)
            self.writeCsv(item=self.content,siteName=j)
            self.content.clear()
        self.itemName1 = []
        self.itemName2 = []
        self.itemName3 = []
        self.itemName4 = []
        self.itemName5 = []
        self.itemName6 = []
        self.itemName7 = []
        self.token = []
        time2 = time.time()
        time3 = time2-time1
        print 'script consume time:', time2-time1
        logging.info('the spider running time is: %s seconds'%str(time3))

        # print len(self.content)

if __name__ == "__main__":
    # sched =BlockingScheduler()
    # sched.add_job(draker().start,'interval',minutes = 5,start_date ='2017-07-14 14:47:00',end_date = '2017-07-14 16:50:00')
    # sched.start()
    while True:
        draker().start()

        # 获取当前运行的pid
        p1 = psutil.Process(os.getpid())

        # 打印本机的内存信息
        print ('直接打印内存占用： ' + (str)(psutil.virtual_memory))

        # 打印内存的占用率
        print ('获取内存占用率： ' + (str)(psutil.virtual_memory().percent) + '%')

        # 本机cpu的总占用率
        print ('打印本机cpu占用率： ' + (str)(psutil.cpu_percent(0)) + '%')

        # 该进程所占cpu的使用率
        print (" 打印该进程CPU占用率: " + (str)(p1.cpu_percent(None)) + "%")

        # 直接打印进程所占内存占用率
        print (p1.memory_percent)

        # 格式化后显示的进程内存占用率
        print "percent: %.2f%%" % (p1.memory_percent())

        time.sleep(900)
